# -*- coding: utf-8 -*-

import scrapy
import telnetlib
from MyScrapy.items import XiciProxy, JiguangProxy


class XiciProxySpider(scrapy.Spider):
    name = 'XiciProxy'
    proxy_list = []
    allowed_domains = ['xicidaili.com']
    start_urls = ['https://www.xicidaili.com/', ]

    def parse(self, response):
        table = response.xpath('//table[@id="ip_list"]')[0]
        trs = table.xpath('//tr')[1:]   # 去掉标题行
        items = []
        for tr in trs:
            pre_item = XiciProxy()
            pre_item['ip'] = tr.xpath('td[2]/text()').extract()[0]
            pre_item['port'] = tr.xpath('td[3]/text()').extract()[0]
            pre_item['http'] = tr.xpath('td[6]/text()').extract()[0]
            try:
                telnetlib.Telnet(pre_item['ip'], port=pre_item['port'], timeout=2)  # 验证是否可用
            except:
                print('connect failed')
            else:
                address = pre_item['http'] + '://' + pre_item['ip'] + ':' + pre_item['port']
                print('success:' + address)
                items.append(address)
                yield pre_item
        return items

    def start_requests(self):
        res = []
        for i in range(1, 2):
            url = 'https://www.xicidaili.com/nn/%d' % i
            req = scrapy.Request(url)
            # 存储所有对应地址的请求
            res.append(req)
        return res


class JiguangProxySpider(scrapy.Spider):
    proxy_list = []
    name = 'JiguangProxy'
    allowed_domains = ['jghttp.golangapi.com']
    start_urls = ['http://d.jghttp.golangapi.com/getip?num=50&type=1&pro=&city=0&yys=0&port=1&pack=6701&ts=0&ys=0&cs=0&lb=1&sb=0&pb=4&mr=1&regions=']

    def parse(self, response):
        proxy_list = response.text.splitlines()
        print(proxy_list)
        for proxy in proxy_list:
            result = proxy.split(':')
            try:
                telnetlib.Telnet(result[0], port=result[1], timeout=2)  # 验证是否可用
            except:
                print('connect failed')
            else:
                print('success:' + proxy)
                item = JiguangProxy()
                item['address'] = proxy
                yield item

